package com.ai.nlp.news;

import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Baidu {
	ConcurrentLinkedQueue<String> queue = new ConcurrentLinkedQueue<String>();
	BlockingQueue<Map<String, String>> data = new LinkedBlockingQueue<Map<String, String>>();

	private Baidu() {
		queue.offer("http://www.huanqiu.com/");
	}

	private static class Holder {
		private static final Baidu INSTANCE = new Baidu();
	}

	public static Baidu getInstance() {
		return Holder.INSTANCE;
	}

	public void baidu() throws IOException {
		String url = null;
		while (queue.size() > 0) {
			url = queue.poll();
			if (url == null || "".equals(url)) {
				continue;
			}
			if (url.startsWith("http") || url.startsWith("https")) {
				Document doc = null;
				try {
					doc = Jsoup.connect(url).get();
				} catch (Exception e) {
					continue;
				}

				Map<String, String> map = new HashMap<>();
				Elements elements = doc.select("a");
				for (Element element : elements) {

					url = element.attr("href");
					
					if (url.startsWith("http") || url.startsWith("https")) {
						String title = element.ownText();
						// System.out.println(title);
						title = removeTag(title);
						title = title.replaceAll("[\\s\\n \\t]+", "").replaceAll("\\d+", "");
						if (!isEmpty(url)) {
							queue.offer(url);
						}
						if (isEmpty(url) || isEmpty(title)) {
							continue;
						}
					
						map.put("url", url);
						map.put("title", title);
					}

					data.offer(map);
				}

			}
		}

	}

	private boolean isEmpty(String input) {
		if (input == null) {
			return true;
		}
		if ("".equals(input)) {
			return true;
		}
		if (input.length() < 5) {
			return true;
		}
		return false;
	}

	public String removeTag(String htmlStr) {
		String regEx_script = "<script[^>]*?>[\\s\\S]*?<\\/script>"; // script
		String regEx_style = "<style[^>]*?>[\\s\\S]*?<\\/style>"; // style
		String regEx_html = "<[^>]+>"; // HTML tag
		String regEx_space = "\\s+|\t|\r|\n";// other characters

		Pattern p_script = Pattern.compile(regEx_script, Pattern.CASE_INSENSITIVE);
		Matcher m_script = p_script.matcher(htmlStr);
		htmlStr = m_script.replaceAll("");
		Pattern p_style = Pattern.compile(regEx_style, Pattern.CASE_INSENSITIVE);
		Matcher m_style = p_style.matcher(htmlStr);
		htmlStr = m_style.replaceAll("");
		Pattern p_html = Pattern.compile(regEx_html, Pattern.CASE_INSENSITIVE);
		Matcher m_html = p_html.matcher(htmlStr);
		htmlStr = m_html.replaceAll("");
		Pattern p_space = Pattern.compile(regEx_space, Pattern.CASE_INSENSITIVE);
		Matcher m_space = p_space.matcher(htmlStr);
		htmlStr = m_space.replaceAll(" ");
		return htmlStr;
	}

	public void write() {
		Set<String> f = new HashSet<>();
		while (true) {
			FileOutputStream fos = null;
			FileChannel channel = null;
			try {

				fos = new FileOutputStream("E:\\log\\baidu.log", true);
				channel = fos.getChannel();
				List<Map<String, String>> list = new ArrayList<>(1000);
				data.drainTo(list, 1000);
				StringBuilder data = new StringBuilder();

				for (Map<String, String> map : list) {
					String url = map.get("url");
					String title = map.get("title");
					if (isEmpty(url) || isEmpty(title) || f.contains(url)) {
						continue;
					}
					f.add(url);
					data.append(url).append("####").append(title).append("\n");
				}
				ByteBuffer buffer = ByteBuffer.allocate(data.capacity() * 2);
				buffer.put(data.toString().getBytes());
				buffer.flip();
				try {
					channel.write(buffer);
				} catch (IOException e) {
					e.printStackTrace();
				}
			} catch (Exception e) {
				e.printStackTrace();
			} finally {
				try {
					if (channel != null) {
						channel.close();
					}
					if (fos != null) {
						fos.close();
					}
				} catch (IOException e) {
					e.printStackTrace();
				}

			}

		}
	}

	public static void main(String[] args) throws IOException {
		int cup = Runtime.getRuntime().availableProcessors();
		System.out.println(cup);
		int threadCount = cup*2 +1;
		CountDownLatch latch = new CountDownLatch(threadCount);
		ExecutorService es = Executors.newFixedThreadPool(threadCount);
		for (int i = 0; i < (threadCount-2); i++) {
			es.execute(() -> {
				try {
					Baidu.getInstance().baidu();
				} catch (IOException e) {
					e.printStackTrace();
				}
				latch.countDown();
			});
		}

		for (int i = 0; i < 2; i++) {
			es.execute(() -> {
				Baidu.getInstance().write();
				latch.countDown();
			});
		}
		try {
			latch.await(10, TimeUnit.MINUTES);
			es.shutdown();
		} catch (InterruptedException e) {
			e.printStackTrace();
			es.shutdownNow();
		}
	}

}
